package engine

import (
	"learngo/crawler/fetch"
	"log"
)

func Run(seed ...Request){
	var requests []Request
	for _,r:=range seed{
		requests=append(requests,r)
	}
	for len(requests)>0 {
		r :=requests[0]
		requests=requests[1:]
		log.Printf("Fetch Url : %s\n",r.Url)
		body,err:=fetch.Fetch(r.Url)
		if err!=nil {
			log.Printf("fetch err:%s %s \n",r.Url,err)
			continue
		}
		ParseResult:=r.ParserFunc(body)
		requests=append(requests,ParseResult.Requests...)
		for _,item:=range ParseResult.Items{
			log.Printf("get items %v",item)
		}
	}

}
